package com.jonas.sparkwork.answer1

import org.apache.spark.sql.SparkSession

/**
 * @author Jonas Gao
 * @since 2022/4/16
 */
object Answer1App {
  def main(args: Array[String]): Unit = {
    val session = SparkSession
      .builder()
      .appName("Spark Inverted Index")
      .getOrCreate()
    val rdd0 = session.sparkContext.parallelize(Seq(
      "it is what it is",
      "what is it",
      "it is a banana"
    ))
    val rdd1 = rdd0.zipWithIndex().cache()
    rdd1
      .flatMap(t => {
        val (str, index) = t
        str.split(" ").map((_, Set(index)))
      })
      .reduceByKey((a, b) => a ++ b)
      .foreach(println)
    rdd1
      .flatMap(t => {
        val (str, index) = t
        str.split(" ").map(i => ((i, index), 1))
      })
      .reduceByKey(_ + _)
      .map(t => (t._1._1, List((t._1._2, t._2))))
      .reduceByKey(_ ++ _)
      .foreach(println)
    session.stop()
  }
}
